import json
from PIL import Image, ImageDraw
from PIL import ImageColor
import re

additional_colors = [colorname for (colorname, colorcode) in ImageColor.colormap.items()]

def plot_bounding_boxes(im, noun_phrases_and_positions):
    """
    Plots bounding boxes on an image with markers for each noun phrase, using PIL, normalized coordinates, and different colors.

    Args:
        img_path: The path to the image file.
        noun_phrases_and_positions: A list of tuples containing the noun phrases
         and their positions in normalized [y1 x1 y2 x2] format.
    """

    # Load the image
    img = im
    width, height = img.size
    print(img.size)
    # Create a drawing object
    draw = ImageDraw.Draw(img)

    # Define a list of colors
    colors = [
    'red',
    'green',
    'blue',
    ] + additional_colors

    # Iterate over the noun phrases and their positions
    for i, (noun_phrase, (y1, x1, y2, x2)) in enumerate(
        noun_phrases_and_positions):
        # Select a color from the list
        color = colors[i % len(colors)]

        # Convert normalized coordinates to absolute coordinates
        abs_x1 = int(x1/1000 * width)
        abs_y1 = int(y1/1000 * height)
        abs_x2 = int(x2/1000 * width)
        abs_y2 = int(y2/1000 * height)

        # Draw the bounding box
        draw.rectangle(
            ((abs_x1, abs_y1), (abs_x2, abs_y2)), outline=color, width=4
        )

        # Draw the text
        draw.text((abs_x1 + 8, abs_y1 + 6), noun_phrase, fill=color)

    # Display the image

    img.show()

# @title Parsing utils
def parse_list_boxes(text):
  result = []
  for line in text.strip().splitlines():
    # Extract the numbers from the line, remove brackets and split by comma
    try:
      numbers = line.split('[')[1].split(']')[0].split(',')
    except:
      numbers =  line.split('- ')[1].split(',')

    # Convert the numbers to integers and append to the result
    result.append([int(num.strip()) for num in numbers])

  return result



def parse_json_in_output(output):
    """
    Extracts and converts JSON-like data from the given text output to a Python dictionary.
    
    Args:
        output (str): The text output containing the JSON data.
    
    Returns:
        dict: The parsed JSON data as a Python dictionary.
    """
    # Regex to extract JSON-like portion
    json_match = re.search(r"\{.*?\}", output, re.DOTALL)
    if json_match:
        json_str = json_match.group(0)
        # Fix single quotes and ensure proper JSON formatting
        json_str = json_str.replace("'", '"')  # Replace single quotes with double quotes
        try:
            # Convert the fixed JSON string into a dictionary
            json_data = json.loads(json_str)
            return json_data
        except json.JSONDecodeError:
            return "The extracted JSON is still not valid after formatting."
    else:
        return "No JSON data found in the given output."

hospital_image = Image.open("hospital_scene.jpeg")
hospital_image

prescription_image = Image.open("prescription1.jpg")
prescription_image_path ="prescription1.jpg"

prescription_image

json_format = {
                "type": "json_schema",
                "json_schema": {
                    "name": "img_extract",
                    "schema": {
                    "type": "object",
                    "properties": {
                          "full_text": {
                                "type": "string",
                                "description": "the text that is writen on the prescription",
                                "minimum": 0
                                },
                            "drug": {
                                "type": "string",
                                "description": "You should find a drug name on the prescription, per example Dafalgan, Acetaminophen, etc."
                                },
                            "drug_ingestion_information": {
                                "type": "string",
                                "description": "There should be an information related to the frequency of which the drug has to be taken"
                                },
                            "sender": {
                                "type": "string",
                                "description": "There should be an information related to the sender of the prescription, like the name of a doctor or an hospital"
                                },
                            "recipient": {
                                "type": "string",
                                "description": "There should be an information related to the recipient of the prescription, liek a patient name"
                                },
                            "recipient_age":{
                                "type": "integer",
                                "description": "Try to find the age or date of birth of the recipient and return the age as today"
                                },
                       
                        
                    },
                    "required": ["full_text","drug","drug_ingestion_information","sender","recipient","recipient_age"]
                    }}
}

import openai
from dotenv import load_dotenv  
import os
import base64
import json
import textwrap

# Function to encode the image
def encode_image(image_path):
  with open(image_path, "rb") as image_file:
    return base64.b64encode(image_file.read()).decode('utf-8')


load_dotenv()
#openAIclient = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
openAIclient = openai.OpenAI(api_key= os.getenv("OPENAI_API_KEY"))




TEXTMODEL = "gpt-4o-mini" 
IMGMODEL= "gpt-4o-mini"

def promptLLM(prompt : str = None, sysprompt : str = None,  image : str = None, wantJson : bool = False, returnDict : bool = False):
    returnValue = ""
    messages = [{"role": "system", "content" : sysprompt}]
    modelToUse = TEXTMODEL
    #force it to be a json answer prompt
    #prompt = prompt if not wantJson else returnJSONAnswerPrompt(prompt)
    messages.append({"role": "user", "content": [{ 
        "type" : "text", 
        "text" : prompt 
    }]})
    if image is not None:
        image = f"data:image/jpeg;base64,{image}"
        messages[1]["content"].append({"type": "image_url", "image_url": { "url" : image}})
        modelToUse = IMGMODEL

    if wantJson:
        returnValue = openAIclient.chat.completions.create(
            model=modelToUse,
            #max_tokens= 400,
            response_format=json_format,
            messages=messages,
            temperature=0,
            #n=1,
        )
    else :
        returnValue = openAIclient.chat.completions.create(
            model=modelToUse,
            messages=messages,
            temperature=0,
            #n=1,
        )
    returnValue = returnValue.choices[0].message.content
    if returnDict:
        return json.loads(returnValue)
    return returnValue

output_image_analysis = promptLLM(prompt = "describe the image in detail",sysprompt = "you are a careful observer. the response should be in json format", image = encode_image(prescription_image_path), wantJson=True, returnDict=False)

output_image_analysis

'{"full_text":"Dr. Markus Müller\\n\\nClaudia Fischer\\n1.4.1978 (f)\\n\\nIbuprofen\\n3x 400mg\\n(nach dem Essen)\\n\\nKopfstelle","drug":"Ibuprofen","drug_ingestion_information":"3x 400mg (nach dem Essen)","sender":"Dr. Markus Müller","recipient":"Claudia Fischer","recipient_age":45}'

result=parse_json_in_output(output_image_analysis)

drug=result['drug']
drug_ingestion_information=result['drug_ingestion_information']
sender=result['sender']
recipient=result['recipient']
age=result['recipient_age']

result

{'full_text': 'Dr. Markus Müller\n\nClaudia Fischer\n1.4.1978 (f)\n\nIbuprofen\n3x 400mg\n(nach dem Essen)\n\nKopfstelle',
 'drug': 'Ibuprofen',
 'drug_ingestion_information': '3x 400mg (nach dem Essen)',
 'sender': 'Dr. Markus Müller',
 'recipient': 'Claudia Fischer',
 'recipient_age': 45}

import google.generativeai as genai

im = Image.open("hospital_scene.jpeg")
api_key = os.getenv("GEMINI_API_KEY")

genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-1.5-pro")

response = model.generate_content([
    im,
    (
        f"ONLY RETURN THE LABELS AND LIST, NO introduction text. Return bounding boxes for the patienst that are this picture. To help you identify the patient you have the following information: Name of the patient:{recipient}, age of the patient:{age}. Let's assume that women have white hair starting at 60 years old. The label of the box should be the name of the patient:{recipient}, if you can not identify the patient: put unknown patient. ONLY RETURN THE Labels and list"
        " The result should look like this: 'patient_label': [ymin, xmin, ymax, xmax]\n "
    ),
])
response.resolve()
print(response.text)

/home/vscode/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

```json
{
  'Claudia Fischer': [351,676,560,807],
  'unknown patient': [375,200,586,335]
}
```

def parse_list_boxes_with_label(text):
  text = text.split("```\n")[0]
  return json.loads(text.strip("```").strip("python").strip("json").replace("'", '"').replace('\n', '').replace(',}', '}'))

im = hospital_image
boxes = parse_list_boxes_with_label(response.text)
img = plot_bounding_boxes(im, noun_phrases_and_positions=list(boxes.items()))
img=im.save("hospital_scene_patient1_identified.jpeg")

(1792, 1024)

prescription_image_2 = Image.open("extra_case/prescription2.jpg")
prescription_image_path_2 ="extra_case/prescription2.jpg"

prescription_image_2

output_image_analysis = promptLLM(prompt = "describe the image in detail",sysprompt = "you are a careful observer. the response should be in json format", image = encode_image(prescription_image_path_2), wantJson=True, returnDict=False)

output_image_analysis

'{"full_text":"Dr. Katja Fischer\\nChirurgie\\n\\nClaudie Müller\\n1.12. 1944 (F)\\n\\nDafalgan 3x 600mg\\n(nach dem Essen)\\n\\nK. Fischer","drug":"Dafalgan","drug_ingestion_information":"3x 600mg (nach dem Essen)","sender":"Dr. Katja Fischer","recipient":"Claudie Müller","recipient_age":79}'

result=parse_json_in_output(output_image_analysis)

drug=result['drug']
drug_ingestion_information=result['drug_ingestion_information']
sender=result['sender']
recipient=result['recipient']
age=result['recipient_age']

result

{'full_text': 'Dr. Katja Fischer\nChirurgie\n\nClaudie Müller\n1.12. 1944 (F)\n\nDafalgan 3x 600mg\n(nach dem Essen)\n\nK. Fischer',
 'drug': 'Dafalgan',
 'drug_ingestion_information': '3x 600mg (nach dem Essen)',
 'sender': 'Dr. Katja Fischer',
 'recipient': 'Claudie Müller',
 'recipient_age': 79}

import google.generativeai as genai

im = Image.open("hospital_scene.jpeg")
api_key = os.getenv("GEMINI_API_KEY")

genai.configure(api_key=api_key)
model = genai.GenerativeModel("gemini-1.5-pro")

response = model.generate_content([
    im,
    (
        f"ONLY RETURN THE LABELS AND LIST, NO introduction text. Return bounding boxes for the patienst that are this picture. To help you identify the patient you have the following information: Name of the patient:{recipient}, age of the patient:{age}. Let's assume that women have white hair starting at 60 years old. The label of the box should be the name of the patient:{recipient}, if you can not identify the patient: put unknown patient. ONLY RETURN THE Labels and list"
        " The result should look like this: 'patient_label': [ymin, xmin, ymax, xmax]\n "
    ),
])
response.resolve()
print(response.text)

```json
{
  'Claudie Müller': [372,195,597,335]
}
```

im = Image.open("hospital_scene.jpeg")
boxes = parse_list_boxes_with_label(response.text)
img = plot_bounding_boxes(im, noun_phrases_and_positions=list(boxes.items()))
img=im.save("hospital_scene_patient2_identified.jpeg")

(1792, 1024)

import os
import platform
import socket
from platform import python_version
from datetime import datetime

print('-----------------------------------')
print(os.name.upper())
print(platform.system(), '|', platform.release())
print('Datetime:', datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('Python Version:', python_version())
print('IP Address:', socket.gethostbyname(socket.gethostname()))
print('-----------------------------------')

-----------------------------------
POSIX
Linux | 6.5.0-1025-azure
Datetime: 2024-12-16 10:25:08
Python Version: 3.10.16
IP Address: 127.0.0.1
-----------------------------------

Vorname:	Mauricio
Nachname:	Saez
Immatrikulationsnummer:	12642112
Modul:	Data Science
Prüfungsdatum / Raum / Zeit:	16.12.2024 / Raum: SF O3.54 / 8:00 – 12:30
Erlaubte Hilfsmittel:	w.MA.XX.DS.24HS (Data Science) Open Book, Eigener Computer, Internet-Zugang
Nicht erlaubt:	Nicht erlaubt ist der Einsatz beliebiger Formen von generativer KI (z.B. Copilot, ChatGPT) sowie beliebige Formen von Kommunikation oder Kollaboration mit anderen Menschen.

Kategorie	Beschreibung	Punkteverteilung
Code nicht lauffähig oder Ergebnisse nicht relevant	Der Code läuft nicht oder erfüllt nicht die Anforderungen des Aspekts (z. B. Bilder werden nicht geladen, die Textausgabe der Extraktion fehlt, Bounding Boxes werden nicht angezeigt).	0 Punkte
Code lauffähig, aber mit gravierenden Mängeln	Der Code läuft, jedoch fehlen zentrale Teile der Funktionalität eines Aspekts (z. B. unvollständige Extraktion von Bildinformationen oder Fehler bei der Definition eines Schemas).	25% der max. erreichbaren Punkte
Code lauffähig, aber mit mittleren Mängeln	Der Code läuft und liefert teilweise korrekte Ergebnisse für einen Aspekt, aber wichtige Details fehlen (z. B. ungenaue Bounding Boxes, unvollständige Integration der extrahierten Daten).	50% der max. erreichbaren Punkte
Code lauffähig, aber mit minimalen Mängeln	Der Code erfüllt die Anforderungen eines Aspekts weitgehend, aber kleinere Fehler oder Abweichungen (z. B. nicht robust Extraktionsdaten, kleinere Schemaabweichungen, Prompt zu wenig stringent formuliert -> teilweise unstabile Output)sind vorhanden.	75% der max. erreichbaren Punkte
Code lauffähig und korrekt	Der Code erfüllt die Anforderungen des Aspekts vollständig und liefert die erwarteten Ergebnisse ohne Fehler (z. B. korrekte Extraktion, vollständige Bounding Boxes, saubere Integration).	100% der max. erreichbaren Punkte

Arbeitspaket (AP) 3: Multimodal Interactions - Images¶

Persönliche Angaben (bitte ergänzen)¶

Bewertungskriterien¶

Vorbereitung (Dieser Teil wird nicht bewertet!)¶

Aufgabe (Dieser Teil wird bewertet!)¶

Hinweise zum folgenden Arbeitspaket:¶

Utils (Hilfsfunktionen):¶

Aufgabe (1): Laden und Visualisieren von Bildern / Importing notwendige Libraries¶

Aufgabe (2): Definition eines strukturierten Schemas für die Extraktion von Informationen¶

Aufgabe (3): Extraktion von visuellen Informationen mittels Prompting¶

Aufgabe (4): Verarbeitung der textuellen Ausgabe der visuellen Extraktion¶

Aufgabe (5): Integration der extrahierten Daten in neuem Prompt für weitere Identifizierung¶

Aufgabe (6): Visualisierung von Bounding Boxes¶

Aufgabe (7): Pipeline Validation¶

Jupyter notebook --footer info-- (please always provide this at the end of each notebook)¶